/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jobset

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2"

	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2"
	controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants"
	"sigs.k8s.io/kueue/pkg/controller/jobframework"
	"sigs.k8s.io/kueue/pkg/features"
	utiltesting "sigs.k8s.io/kueue/pkg/util/testing"
	utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2"
	testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset"
)

func TestPodsReady(t *testing.T) {
	testcases := map[string]struct {
		jobSet jobset.JobSet
		want   bool
	}{
		"all jobs are ready": {
			jobSet: *testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    2,
					Parallelism: 1,
					Completions: 1,
				},
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-2",
					Replicas:    3,
					Parallelism: 1,
					Completions: 1,
				},
			).JobsStatus(
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-1",
					Ready:     1,
					Succeeded: 1,
				},
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-2",
					Ready:     3,
					Succeeded: 0,
				},
			).Obj(),
			want: true,
		},
		"not all jobs are ready": {
			jobSet: *testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    2,
					Parallelism: 1,
					Completions: 1,
				},
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-2",
					Replicas:    3,
					Parallelism: 1,
					Completions: 1,
				},
			).JobsStatus(
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-1",
					Ready:     1,
					Succeeded: 0,
				},
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-2",
					Ready:     1,
					Succeeded: 2,
				},
			).Obj(),
			want: false,
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			jobSet := (JobSet)(tc.jobSet)
			ctx, _ := utiltesting.ContextWithLog(t)
			got := jobSet.PodsReady(ctx)
			if tc.want != got {
				t.Errorf("Unexpected response (want: %v, got: %v)", tc.want, got)
			}
		})
	}
}

func TestReclaimablePods(t *testing.T) {
	baseWrapper := testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
		testingjobset.ReplicatedJobRequirements{
			Name:        "replicated-job-1",
			Replicas:    1,
			Parallelism: 2,
			Completions: 2,
		},
		testingjobset.ReplicatedJobRequirements{
			Name:        "replicated-job-2",
			Replicas:    2,
			Parallelism: 3,
			Completions: 6,
		},
	)

	testcases := map[string]struct {
		jobSet *jobset.JobSet
		want   []kueue.ReclaimablePod
	}{
		"no status": {
			jobSet: baseWrapper.Clone().Obj(),
			want:   nil,
		},
		"empty jobs status": {
			jobSet: baseWrapper.Clone().JobsStatus().Obj(),
			want:   nil,
		},
		"single job done": {
			jobSet: baseWrapper.Clone().JobsStatus(jobset.ReplicatedJobStatus{
				Name:      "replicated-job-1",
				Succeeded: 1,
			}).Obj(),
			want: []kueue.ReclaimablePod{{
				Name:  "replicated-job-1",
				Count: 2,
			}},
		},
		"single job partial done": {
			jobSet: baseWrapper.Clone().JobsStatus(jobset.ReplicatedJobStatus{
				Name:      "replicated-job-2",
				Succeeded: 1,
			}).Obj(),
			want: []kueue.ReclaimablePod{{
				Name:  "replicated-job-2",
				Count: 3,
			}},
		},
		"all done": {
			jobSet: baseWrapper.Clone().JobsStatus(
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-1",
					Succeeded: 1,
				},
				jobset.ReplicatedJobStatus{
					Name:      "replicated-job-2",
					Succeeded: 2,
				},
			).Obj(),
			want: []kueue.ReclaimablePod{
				{
					Name:  "replicated-job-1",
					Count: 2,
				},
				{
					Name:  "replicated-job-2",
					Count: 6,
				},
			},
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			ctx, _ := utiltesting.ContextWithLog(t)
			jobSet := (*JobSet)(tc.jobSet)
			got, err := jobSet.ReclaimablePods(ctx)
			if err != nil {
				t.Fatalf("Unexpected error: %s", err)
			}
			if diff := cmp.Diff(tc.want, got); diff != "" {
				t.Errorf("Unexpected Reclaimable pods (-want +got):\n%s", diff)
			}
		})
	}
}

func TestPodSets(t *testing.T) {
	jobSetTemplate := testingjobset.MakeJobSet("jobset", "ns")

	testCases := map[string]struct {
		jobSet                        *JobSet
		wantPodSets                   func(jobSet *JobSet) []kueue.PodSet
		enableTopologyAwareScheduling bool
	}{
		"no annotations": {
			jobSet: (*JobSet)(jobSetTemplate.Clone().
				ReplicatedJobs(
					testingjobset.ReplicatedJobRequirements{Name: "job1", Replicas: 2, Parallelism: 1, Completions: 1},
					testingjobset.ReplicatedJobRequirements{Name: "job2", Replicas: 3, Parallelism: 2, Completions: 3},
				).
				Obj()),
			wantPodSets: func(jobSet *JobSet) []kueue.PodSet {
				return []kueue.PodSet{
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[0].Name), 2).
						PodSpec(*jobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.DeepCopy()).
						Obj(),
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[1].Name), 6).
						PodSpec(*jobSet.Spec.ReplicatedJobs[1].Template.Spec.Template.Spec.DeepCopy()).
						Obj(),
				}
			},
			enableTopologyAwareScheduling: false,
		},
		"with required topology annotation": {
			jobSet: (*JobSet)(jobSetTemplate.Clone().
				ReplicatedJobs(
					testingjobset.ReplicatedJobRequirements{
						Name:        "job1",
						Replicas:    2,
						Parallelism: 1,
						Completions: 1,
						PodAnnotations: map[string]string{
							kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block",
						},
					},
					testingjobset.ReplicatedJobRequirements{Name: "job2", Replicas: 3, Parallelism: 2, Completions: 3},
				).
				Obj()),
			wantPodSets: func(jobSet *JobSet) []kueue.PodSet {
				return []kueue.PodSet{
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[0].Name), 2).
						PodSpec(*jobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.DeepCopy()).
						Annotations(map[string]string{kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block"}).
						RequiredTopologyRequest("cloud.com/block").
						PodIndexLabel(ptr.To(batchv1.JobCompletionIndexAnnotation)).
						SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
						SubGroupCount(ptr.To[int32](2)).
						Obj(),
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[1].Name), 6).
						PodSpec(*jobSet.Spec.ReplicatedJobs[1].Template.Spec.Template.Spec.DeepCopy()).
						PodIndexLabel(ptr.To(batchv1.JobCompletionIndexAnnotation)).
						SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
						SubGroupCount(ptr.To[int32](3)).
						Obj(),
				}
			},
			enableTopologyAwareScheduling: true,
		},
		"with preferred topology annotation": {
			jobSet: (*JobSet)(jobSetTemplate.Clone().
				ReplicatedJobs(
					testingjobset.ReplicatedJobRequirements{Name: "job1", Replicas: 2, Parallelism: 1, Completions: 1},
					testingjobset.ReplicatedJobRequirements{
						Name:        "job2",
						Replicas:    3,
						Parallelism: 2,
						Completions: 3,
						PodAnnotations: map[string]string{
							kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block",
						},
					},
				).
				Obj()),
			wantPodSets: func(jobSet *JobSet) []kueue.PodSet {
				return []kueue.PodSet{
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[0].Name), 2).
						PodSpec(*jobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.DeepCopy()).
						PodIndexLabel(ptr.To(batchv1.JobCompletionIndexAnnotation)).
						SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
						SubGroupCount(ptr.To[int32](2)).
						Obj(),
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[1].Name), 6).
						PodSpec(*jobSet.Spec.ReplicatedJobs[1].Template.Spec.Template.Spec.DeepCopy()).
						Annotations(map[string]string{kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block"}).
						PreferredTopologyRequest("cloud.com/block").
						PodIndexLabel(ptr.To(batchv1.JobCompletionIndexAnnotation)).
						SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
						SubGroupCount(ptr.To[int32](3)).
						Obj(),
				}
			},
			enableTopologyAwareScheduling: true,
		},
		"without required and preferred topology annotation if TAS is disabled": {
			jobSet: (*JobSet)(jobSetTemplate.Clone().
				ReplicatedJobs(
					testingjobset.ReplicatedJobRequirements{Name: "job1", Replicas: 2, Parallelism: 1, Completions: 1},
					testingjobset.ReplicatedJobRequirements{
						Name:        "job2",
						Replicas:    2,
						Parallelism: 1,
						Completions: 1,
						PodAnnotations: map[string]string{
							kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block",
						},
					},
					testingjobset.ReplicatedJobRequirements{
						Name:        "job3",
						Replicas:    3,
						Parallelism: 2,
						Completions: 3,
						PodAnnotations: map[string]string{
							kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block",
						},
					},
				).
				Obj()),
			wantPodSets: func(jobSet *JobSet) []kueue.PodSet {
				return []kueue.PodSet{
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[0].Name), 2).
						PodSpec(*jobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.DeepCopy()).
						Obj(),
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[1].Name), 2).
						PodSpec(*jobSet.Spec.ReplicatedJobs[1].Template.Spec.Template.Spec.DeepCopy()).
						Annotations(map[string]string{kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block"}).
						Obj(),
					*utiltestingapi.MakePodSet(kueue.NewPodSetReference(jobSet.Spec.ReplicatedJobs[2].Name), 6).
						PodSpec(*jobSet.Spec.ReplicatedJobs[2].Template.Spec.Template.Spec.DeepCopy()).
						Annotations(map[string]string{kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block"}).
						Obj(),
				}
			},
			enableTopologyAwareScheduling: false,
		},
	}
	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			features.SetFeatureGateDuringTest(t, features.TopologyAwareScheduling, tc.enableTopologyAwareScheduling)
			ctx, _ := utiltesting.ContextWithLog(t)
			gotPodSets, err := tc.jobSet.PodSets(ctx)
			if err != nil {
				t.Fatalf("unexpected error: %s", err)
			}
			if diff := cmp.Diff(tc.wantPodSets(tc.jobSet), gotPodSets); diff != "" {
				t.Errorf("pod sets mismatch (-want +got):\n%s", diff)
			}
		})
	}
}

var (
	jobCmpOpts = cmp.Options{
		cmpopts.EquateEmpty(),
		cmpopts.IgnoreFields(jobset.JobSet{}, "TypeMeta", "ObjectMeta"),
	}
	workloadCmpOpts = cmp.Options{
		cmpopts.EquateEmpty(),
		cmpopts.IgnoreFields(kueue.Workload{}, "TypeMeta"),
		cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name", "Labels", "ResourceVersion", "OwnerReferences", "Finalizers"),
		cmpopts.IgnoreFields(kueue.WorkloadSpec{}, "Priority"),
		cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime"),
		cmpopts.IgnoreFields(kueue.PodSet{}, "Template"),
	}
)

func TestReconciler(t *testing.T) {
	baseWPCWrapper := utiltestingapi.MakeWorkloadPriorityClass("test-wpc").
		PriorityValue(100)
	basePCWrapper := utiltesting.MakePriorityClass("test-pc").
		PriorityValue(200)

	testNamespace := utiltesting.MakeNamespaceWrapper("ns").Label(corev1.LabelMetadataName, "ns").Obj()

	cases := map[string]struct {
		reconcilerOptions []jobframework.Option
		job               *jobset.JobSet
		priorityClasses   []client.Object
		wantJob           *jobset.JobSet
		wantWorkloads     []kueue.Workload
		wantErr           error
	}{
		"workload is created with podsets and a ProvReq annotation": {
			reconcilerOptions: []jobframework.Option{
				jobframework.WithManageJobsWithoutQueueName(true),
				jobframework.WithManagedJobsNamespaceSelector(labels.Everything()),
			},
			job: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-2",
					Replicas:    2,
					Completions: 2,
					Parallelism: 2,
				}).
				Annotations(map[string]string{
					controllerconsts.ProvReqAnnotationPrefix + "test-annotation": "test-val",
					"invalid-provreq-prefix/test-annotation-2":                   "test-val-2",
				}).
				Obj(),
			wantJob: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-2",
					Replicas:    2,
					Completions: 2,
					Parallelism: 2,
				}).
				Annotations(map[string]string{
					controllerconsts.ProvReqAnnotationPrefix + "test-annotation": "test-val",
					"invalid-provreq-prefix/test-annotation-2":                   "test-val-2",
				}).
				Obj(),
			wantWorkloads: []kueue.Workload{
				*utiltestingapi.MakeWorkload("jobset", "ns").
					Annotations(map[string]string{controllerconsts.ProvReqAnnotationPrefix + "test-annotation": "test-val"}).
					PodSets(
						*utiltestingapi.MakePodSet("replicated-job-1", 1).
							PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")).
							SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
							SubGroupCount(ptr.To[int32](1)).
							Obj(),
						*utiltestingapi.MakePodSet("replicated-job-2", 4).
							PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")).
							SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
							SubGroupCount(ptr.To[int32](2)).
							Obj(),
					).
					Obj(),
			},
		},
		"workload is created with podsets and workloadPriorityClass": {
			reconcilerOptions: []jobframework.Option{
				jobframework.WithManageJobsWithoutQueueName(true),
				jobframework.WithManagedJobsNamespaceSelector(labels.Everything()),
			},
			job: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).WorkloadPriorityClass("test-wpc").Obj(),
			priorityClasses: []client.Object{
				baseWPCWrapper.Obj(),
			},
			wantJob: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).WorkloadPriorityClass("test-wpc").Obj(),
			wantWorkloads: []kueue.Workload{
				*utiltestingapi.MakeWorkload("jobset", "ns").
					WorkloadPriorityClassRef("test-wpc").
					Priority(100).
					PodSets(
						*utiltestingapi.MakePodSet("replicated-job-1", 1).
							PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")).
							SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
							SubGroupCount(ptr.To[int32](1)).
							Obj(),
					).
					Obj(),
			},
		},
		"workload is created with podsets and PriorityClass": {
			reconcilerOptions: []jobframework.Option{
				jobframework.WithManageJobsWithoutQueueName(true),
				jobframework.WithManagedJobsNamespaceSelector(labels.Everything()),
			},
			job: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).PriorityClass("test-pc").Obj(),
			priorityClasses: []client.Object{
				basePCWrapper.Obj(),
			},
			wantJob: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).PriorityClass("test-pc").Obj(),
			wantWorkloads: []kueue.Workload{
				*utiltestingapi.MakeWorkload("jobset", "ns").
					PodPriorityClassRef("test-pc").
					Priority(200).
					PodSets(
						*utiltestingapi.MakePodSet("replicated-job-1", 1).
							PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")).
							SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
							SubGroupCount(ptr.To[int32](1)).
							Obj(),
					).
					Obj(),
			},
		},
		"workload is created with podsets, workloadPriorityClass and PriorityClass": {
			reconcilerOptions: []jobframework.Option{
				jobframework.WithManageJobsWithoutQueueName(true),
				jobframework.WithManagedJobsNamespaceSelector(labels.Everything()),
			},
			job: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).PriorityClass("test-pc").WorkloadPriorityClass("test-wpc").Obj(),
			priorityClasses: []client.Object{
				basePCWrapper.Obj(), baseWPCWrapper.Obj(),
			},
			wantJob: testingjobset.MakeJobSet("jobset", "ns").ReplicatedJobs(
				testingjobset.ReplicatedJobRequirements{
					Name:        "replicated-job-1",
					Replicas:    1,
					Completions: 1,
					Parallelism: 1,
				},
			).PriorityClass("test-pc").WorkloadPriorityClass("test-wpc").Obj(),
			wantWorkloads: []kueue.Workload{
				*utiltestingapi.MakeWorkload("jobset", "ns").
					WorkloadPriorityClassRef("test-wpc").
					Priority(100).
					PodSets(
						*utiltestingapi.MakePodSet("replicated-job-1", 1).
							PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")).
							SubGroupIndexLabel(ptr.To(jobset.JobIndexKey)).
							SubGroupCount(ptr.To[int32](1)).
							Obj(),
					).
					Obj(),
			},
		},
	}

	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			ctx, _ := utiltesting.ContextWithLog(t)
			clientBuilder := utiltesting.NewClientBuilder(jobset.AddToScheme)
			indexer := utiltesting.AsIndexer(clientBuilder)
			if err := SetupIndexes(ctx, indexer); err != nil {
				t.Fatalf("Could not setup indexes: %v", err)
			}
			objs := append(tc.priorityClasses, tc.job, testNamespace)
			kClient := clientBuilder.WithObjects(objs...).Build()
			recorder := record.NewBroadcaster().NewRecorder(kClient.Scheme(), corev1.EventSource{Component: "test"})
			reconciler, err := NewReconciler(ctx, kClient, indexer, recorder, tc.reconcilerOptions...)
			if err != nil {
				t.Errorf("Error creating the reconciler: %v", err)
			}

			jobKey := client.ObjectKeyFromObject(tc.job)
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: jobKey,
			})
			if diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()); diff != "" {
				t.Errorf("Reconcile returned error (-want,+got):\n%s", diff)
			}

			var gotJobSet jobset.JobSet
			if err := kClient.Get(ctx, jobKey, &gotJobSet); err != nil {
				t.Fatalf("Could not get Job after reconcile: %v", err)
			}
			if diff := cmp.Diff(tc.wantJob, &gotJobSet, jobCmpOpts...); diff != "" {
				t.Errorf("Job after reconcile (-want,+got):\n%s", diff)
			}
			var gotWorkloads kueue.WorkloadList
			if err := kClient.List(ctx, &gotWorkloads); err != nil {
				t.Fatalf("Could not get Workloads after reconcile: %v", err)
			}
			if diff := cmp.Diff(tc.wantWorkloads, gotWorkloads.Items, workloadCmpOpts...); diff != "" {
				t.Errorf("Workloads after reconcile (-want,+got):\n%s", diff)
			}
		})
	}
}
